{
 "cells": [
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-03T01:38:39.462575Z",
     "start_time": "2024-06-03T01:38:39.276186Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import requests\n",
    "from lxml import etree\n",
    "import pandas as pd"
   ],
   "id": "initial_id",
   "execution_count": 1,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "num = 0\n",
    "url = \"https://book.douban.com/tag/%E5%B0%8F%E8%AF%B4?start={}&type=T\"\n",
    "headers = {\"User-Agent\": \"Mozilla/5.0 (Windows NT 6.1; WOW64)\"\n",
    "                         \" AppleWebKit/537.36 (KHTML, like Gecko)\"\n",
    "                         \" Chrome/67.0.3396.99 Safari/537.36\"}\n",
    "\n",
    "index = 0\n",
    "book_name_arr = [0] * 1500\n",
    "book_grade_arr = [0] * 1500\n",
    "book_author_arr = [0] * 1500\n",
    "\n",
    "while num < 1500:\n",
    "    url = url.format(num)\n",
    "    html_str = requests.get(url, headers=headers).content.decode()\n",
    "    ret1 = etree.HTML(html_str).xpath(\"//li[@class='subject-item']\")\n",
    "    try:\n",
    "        for content in ret1:\n",
    "            book_name = \"\"\n",
    "            book_name += content.xpath(\".//div[@class='info']/h2/a/text()\")[0].strip()\n",
    "            grade = \"\"\n",
    "            grade += content.xpath(\".//span[@class='rating_nums']/text()\")[0]\n",
    "            author = content.xpath(\".//div[@class='pub']/text()\")[0].split(\"/\")[0].strip()\n",
    "            if index >= 1500:\n",
    "                continue\n",
    "            book_name_arr[index] = book_name\n",
    "            book_grade_arr[index] = grade\n",
    "            book_author_arr[index] = author\n",
    "            index += 1\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "    num += 20\n",
    "\n",
    "df1 = pd.DataFrame({\"名称\": book_name_arr, \"作者\": book_author_arr})\n",
    "df1.to_excel(\"book.xlsx\", index=False, columns=['名称', '作者'])\n",
    "\n",
    "df2 = pd.DataFrame({\"作者\": book_author_arr, \"评分\": book_grade_arr})\n",
    "df2.to_excel(\"grade.xlsx\", index=False, columns=['作者', '评分'])"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-06-03T01:41:56.193856Z",
     "start_time": "2024-06-03T01:40:49.781513Z"
    }
   },
   "id": "ce5de101828113f2",
   "execution_count": 2,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "book = pd.read_excel(\"book.xlsx\")\n",
    "book.head()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-06-03T01:42:06.165497Z",
     "start_time": "2024-06-03T01:42:06.123239Z"
    }
   },
   "id": "31aad59989ae5484",
   "execution_count": 3,
   "outputs": []
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-03T01:42:12.480968Z",
     "start_time": "2024-06-03T01:42:12.440481Z"
    }
   },
   "cell_type": "code",
   "source": [
    "grade = pd.read_excel(\"grade.xlsx\")\n",
    "grade.head()"
   ],
   "id": "e052142c4731f69a",
   "execution_count": 4,
   "outputs": []
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-03T01:42:18.488175Z",
     "start_time": "2024-06-03T01:42:18.252895Z"
    }
   },
   "cell_type": "code",
   "source": [
    "df = pd.merge(book, grade, on='作者')\n",
    "# 直方图\n",
    "df[\"评分\"].hist(bins=20)"
   ],
   "id": "dc2e617c07068cda",
   "execution_count": 5,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# 数据分析\n",
    "ratings = df.groupby(\"作者\")[\"评分\"].count().reset_index(name='评分次数')\n",
    "ratings.sort_values('评分次数', ascending=False).head()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-06-03T01:42:25.986600Z",
     "start_time": "2024-06-03T01:42:25.962270Z"
    }
   },
   "id": "9aa830235a7db757",
   "execution_count": 6,
   "outputs": []
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-03T01:42:37.530219Z",
     "start_time": "2024-06-03T01:42:37.488901Z"
    }
   },
   "cell_type": "code",
   "source": [
    "df['评分'] = pd.to_numeric(df['评分'], errors='coerce')\n",
    "author_book = df.pivot_table(index=\"作者\", columns='名称', values='评分')\n",
    "author_book.fillna(0, inplace=True)\n",
    "author_book"
   ],
   "id": "701dcd0da021e437",
   "execution_count": 7,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# 协同相关性\n",
    "FG = author_book['三体']\n",
    "corr_FG = author_book.corrwith(FG)\n",
    "similarity = pd.DataFrame(corr_FG, columns=['相关系数'])\n",
    "similarity.head()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-06-03T01:45:55.028878Z",
     "start_time": "2024-06-03T01:45:55.015567Z"
    }
   },
   "id": "fa63786a04f655ec",
   "execution_count": 11,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# 智能推荐\n",
    "similarity_new = similarity.join(ratings['评分次数']).fillna(0)\n",
    "similarity_new[similarity_new['评分次数'] >= 0].sort_values(by='相关系数', ascending=False).head()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-06-03T01:45:59.769100Z",
     "start_time": "2024-06-03T01:45:59.759352Z"
    }
   },
   "id": "74f94366d9a8c1b3",
   "execution_count": 12,
   "outputs": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
